#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import division
import os
import sys
import urllib2
import re
import rfc822
from datetime import datetime
from xml.dom.minidom import parseString, getDOMImplementation

def feed_sources():
    f = open("%s/feed_sources.txt" % sys.path[0], 'r')
    urls = map(lambda u: u.strip(), f.readlines())
    f.close()
    if len(urls) > 0:
        return urls

    raise NameError("no urls defined")

def read_source(url):
    try:
        f = urllib2.urlopen(url, None)
    except urllib2.URLError:
        return None
    except urllib2.HTTPError:
        return None
    
    data = f.read()
    f.close()
    return parseString(data)


class Item:
    def __init__(self, id, title, link, description, pub_date, guid):
        self.id = id
        if pub_date:
            try:
                self.dtime = datetime(*rfc822.parsedate_tz(pub_date)[0:6])
            except:
                self.dtime = datetime.now()
        else:
          self.dtime = datetime.now()

        self.title = title.strip()
        self.link = link.strip()
        self.description = description.strip()
        self.pub_date = pub_date
        self.guid = guid
        re_expr = '(?u)\W+'
        s = "%s %s" % (self.title, self.description) 
        re_str = s.lower()\
                .replace('&auml;', u'ä')\
                .replace('&ouml;', u'ö') # there must be a better way ..
        
        self.keywords = set()
        for k in set(re.split(re_expr, re_str)):
            if len(k) > 2:
                self.keywords.add(k.strip())
        
    def __unicode__(self):
        return u"%s" % self.title

    def __str__(self):
        return self.__unicode__()

class Feed:

    def __init__(self, DOM):
        self.DOM = DOM

    def _node_data(self, nodes):
        if len(nodes) != 1: return None
        t = []
        for node in nodes[0].childNodes:
            if node.nodeType == node.TEXT_NODE:
                t.append("%s" % node.data.strip())

        return "".join(t)

    def items(self):
        i = []
        for item in self.DOM.getElementsByTagName("item"):
            i.append(Item(
                    id,
                    self._node_data(item.getElementsByTagName("title")),
                    self._node_data(item.getElementsByTagName("link")),
                    self._node_data(item.getElementsByTagName("description")),
                    self._node_data(item.getElementsByTagName("pubDate")),
                    self._node_data(item.getElementsByTagName("guid"))
                )
            )

        return i

class RSSieve:

    def __init__(self, diff_max, items_count=None):
        self.items = []
        self.items_count = items_count
        self.diff_max = int(diff_max)

    def sorted_items(self):
        return sorted(self.items, key=lambda i: i.dtime, reverse=True)

    def _keyword_diff(self, a, b):
        p = (len(a.difference(b)) / len(a)) * 100
        return 100 - int(p)

    def add_feed(self, feed_items):
        if not self.items:
            self.items = feed_items
            return

        for item in feed_items:
            if item.link in self.links():
                break

            new_item = self.validate_item(item)
            if new_item:
                self.items.append(item)

    def validate_item(self, item):
        for i in self.items:
            diff = self._keyword_diff(item.keywords, i.keywords) 
            if diff > self.diff_max:
                return None
        
        return item

    def links(self):
        return set(l.link for l in self.items)


    def to_rss(self, to_file=None):
        dom = getDOMImplementation().createDocument(None, "rss", None)
        rss = dom.documentElement
        rss.setAttribute('version', '2.0')
        title = dom.createElement('title').appendChild(dom.createTextNode('RSS Suodatin'))
        link = dom.createElement('link').appendChild(dom.createTextNode('www.nurmella.fi/beta/rss.xml'))
        wm = dom.createElement('webMaster').appendChild(dom.createTextNode('peknur@gmail.com'))
        generator = dom.createElement('generator').appendChild(dom.createTextNode('RSSieve'))
        channel = rss.appendChild(dom.createElement('channel'))
        channel.appendChild(title.parentNode)
        channel.appendChild(link.parentNode)
        channel.appendChild(wm.parentNode)
        channel.appendChild(generator.parentNode)
        id = 0
        for i in self.sorted_items():
            item = dom.createElement('item')
            title = dom.createElement('title').appendChild(dom.createTextNode(i.title))
            link = dom.createElement('link').appendChild(dom.createTextNode(i.link))
            description = dom.createElement('description').appendChild(dom.createTextNode(i.description))
            pub_date = dom.createElement('pubDate').appendChild(dom.createTextNode(i.pub_date))
            guid = dom.createElement('guid').appendChild(dom.createTextNode("sID%d" % id))
            item.appendChild(title.parentNode)
            item.appendChild(link.parentNode)
            item.appendChild(description.parentNode)
            item.appendChild(pub_date.parentNode)
            item.appendChild(guid.parentNode)
            channel.appendChild(item)
            id += 1

        if to_file:
            try:
                f = open(to_file, 'w')
                f.write(dom.toxml('utf-8'))
                f.close()
            except IOError:
                pass
              
            return None
        else:
            return dom.toprettyxml()

    def to_json(self):
        pass
      
    def filter_item(self, item):
        pass

if __name__ == '__main__':
    sieve = RSSieve(10)
    if len(sys.argv) == 2:
        output_file = sys.argv[1]
    else:
        output_file = None
    
    tot = 0
    for r in feed_sources():
        dom = read_source(r)
        if dom:
            f = Feed(dom)
            items = f.items()
            tot = tot +  len(items)
            sieve.add_feed(items)

    if output_file:
        sieve.to_rss(output_file)
        os.chmod(output_file, 0750)
    else:
        print sieve.to_rss()
